var request = require('request')
	,fs = require('fs')
	,path = require('path')
	,http = require('http')
	,cheerio = require('cheerio')
	,url = require('url')
	,Iconv = require('iconv-lite')//处理中文编码
	,plugin
	,imgSpider
;
Iconv.extendNodeEncodings();//扩展node 本身的模块
//
//Iconv.undoExtendNodeEncodings();  //解除方法

/*
options:
	total:300 //抓取多少个页面
	dir:'parse3Img/'//保存图片的文件夹
	mainPage:''//类型主页
	downTimeout:20000 //下载(页面或者图片)超时时间
	headers:{ //通用头部
		'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
		,'Cache-Control':'no-cache'
	}

	rule:{ //查找的方法规则
		ruleA:function($){
			return {aList:[],nextUrl:''};
		}
		,ruleImg:function($){
			return [];
		}
	}
*/
function spider(options,rule){
	var _opt = options || {}
		,_optRule = rule || rule
		,_header = { //通用头部
			'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'
			,'Cache-Control':'no-cache'
		}
		,_rule = {
			ruleA:function($){
				return {aList:[],nextUrl:''};
			}
			,ruleImg:function($){
				return [];
			}
		}
	;

	if(!options || !rule) return false;

	this.total = _opt.total || 300; //抓取多少个页面
	this.dir = _opt.dir || 'downLoad/';//下载目录
	this.header = _opt.header || _header;
	this.encoding = _opt.encoding || 'utf8';
	this.mainPage = _opt.mainPage ||　'' ;//类型主页
	this.downTimeout = _opt.downTimeout || 30000; //下载超时时间
	this.runNum = _opt.runNum || 10;//同时下载多少张图片

	this.num = 0;// 第几个网址
	this.aList = [];//某个类型某一页下的所有连接
	this.nextPage = '' ;//当前这个类型分类的下一页的url 比如在类型A里面列举了好几款商品 那么就会有分页，如果当前是第一页那么这个就是第二页的url
	this.nowPage = '' ;//当前网址
	this.isInit = false;
	this._downList = [] ;// 等待下载列表
	this._runList = []; // 正在下载的列表

	this.rule = _optRule ||_rule;

	//this.init();
}

spider.prototype = {
	init:function(){ //初始化
		var _self = this;
		_self.getListPage(_self.mainPage);
	}
	,getPage:function(_url,cb){ //获取页面源码
		console.log(_url+'==');
		var _self = this,_req;
		_req = request({
			url:_url
			,encoding:_self.encoding
			,headers:_self.headers
			,timeout:_self.downTimeout
		}, function (error, response, body) {
		  	if (!error && response.statusCode == 200) {
		  		if(typeof cb === 'function'){
					cb(body);
				}
				response.on('end',function(){
					response.destroy();
					_req.abort();
				});
		  	}else{
		  		console.log(_url+'===fail');
		  		error && (console.log(error));
				response && (console.log(response.statusCode));

		  		console.log('down-list=='+_self._downList.length);
		  	}
		});
	}
	,getPageProxy:function(_url,cb){ //获取页面源码 通过代理服务器获取
		var _self = this, options = {
			hostname: '127.0.0.1',
			port: 8087,
			path: _url
			,headers:_self.headers
			,encoding:_self.encoding
		};
		var req = http.request(options, function(res) {
			var _chunk = '';
			res.setEncoding('utf8');
			res.on('data', function (chunk) {
				_chunk = _chunk + chunk;
			});
			res.on('end',function(){
				if(typeof cb === 'function'){
					cb(_chunk);
				}
				res.destroy();
			});
		});

		req.on('error', function(){
		  console.log('problem with request: ' + e.message);
		});

		req.end();
	}
	,findA:function(chunk,cb){ //找到有效的链接
		var _self = this
			,$ = cheerio.load(chunk)
			,_return = _self.rule.ruleA($)
			,_url = _return.nextUrl
			,nextUrl = ''
			,aList = _return.aList
		;

		//循环链接
		aList.forEach(function(item,index){
			_self.addListA(item);
		});

		if(_url){
			nextUrl = url.resolve(_self.mainPage,_url);
		}

		if(_self.nowPage != nextUrl){
			_self.nextPage = nextUrl;
		}

		if(typeof cb === 'function'){
			cb.call(_self);
		}
	}
	,getImg:function(_url){ //获取图片
		var _self = this;
		_self.getPage(_url,function(chunk){
			_self.findImg(chunk,_self.getImgStep);
		})
	}
	,findImg:function(chunk,cb){ //查找图片
		var _self = this
			,nextUrl=''
			,$ = cheerio.load(chunk)
			,_title=''
			,_dir = _self.dir
			,_titleText = $('title').text()
			,_imgs = _self.rule.ruleImg($)
		;
		_title =  (_titleText.split('-'))[0].trim().replace(/[\n|\/|\\|\||\?|<|>|\*|\:|\"]/g,''); // windows 目录下不支持 "/","\","|","*","?",":","”"，"<",">"

		if(_title){
			_dir = _dir + _title;
		}

		_imgs.forEach(function(item,index){
			_self.saveImg(item,_dir);
		});

		if(typeof cb === 'function'){
			cb.call(_self);
		}
	}
	,saveImg:function(_url,dir){//保存图片
		var _name = path.basename(_url)
			,_self = this
			,_file = dir+'/'+_name
			,_obj = {}
		;
		if(!fs.existsSync(dir) && dir){
			fs.mkdirSync(dir);
		}

		if(!fs.existsSync(_file)){
			_obj = _self.addDownItem(_url,_file);

			_self.checkRunList();
		}
	}
	,checkRunList:function(){
		var _self = this
			,_rList = _self._runList
		;
		//正在下载的数量如果大于设置总数量
		if(_rList.length>=_self.runNum) return false;

		_self.addRunItem();
	}
	,addRunItem:function(){//往执行列表里面添加
		var _self = this
			,_rList = _self._runList
			,_obj
		;
		if(_rList.length<_self.runNum){
			if(_self._downList.length){
				_obj = _self.delDownItem();
				_self._runList.push(_obj);
				_self.downLoadImg(_obj);
			}
		}

		console.log('runList==='+_self._runList.length);
	}
	,delRunItem:function(name){ //在执行列表里面删除
		var _self = this
			,_rList = _self._runList
			,_arr = []
		;
		_rList.forEach(function(item,index){
			if(item!=name){
				_arr.push(item);
			}
		});
		_self._runList = _arr;

		//下载完成一张后再添加一张进去
		_self.addRunItem();
	}
	,addDownItem:function(_url,_file){//添加下载项
		var _self = this
			,name = new String((new Date()).getTime())
			,_list = _self._downList
			,_flag = true
		;
		name.url = _url;
		name.file = _file;

		_list.forEach(function(item,index){
			if(item===name){
				_flag=false;
			}
		});

		if(_flag){//如果不存在在数组中则添加进去
			_self._downList.push(name);
		}else{
			name = null;
		}

		return name;
	}
	,delDownItem:function(name){ //删除下载项
		var _self = this;
		return _self._downList.shift();
	}
	,downLoadImg:function(obj){//原生方法下载图片
		var _self = this
			,_url = obj.url
			,_file = obj.file
			,_req
			,_urlObj = url.parse(_url)
			,_isErr = false
		;

		_req = http.get({
			hostname: _urlObj.hostname
			,port: _urlObj.port || 80
			,path: _urlObj.path
			,header:_self.header
		}, function (res) {
			var imageData ='';
			res.setEncoding('binary');//二进制（binary）
			res.on('data',function(data){//图片加载到内存变量
				imageData += data;
			})
			.on('end',function(){//加载完毕保存图片
				if(!_isErr){
					fs.writeFile(_file, imageData, 'binary', function (err) {//以二进制格式保存
						if (err)throw err;

						_self.delRunItem(obj);
					});
				}

				res.destroy();
				_req.abort();
			})
			;
		});

		_req.on('error',function(err){ //发生错误时 打印错误 并从列表总删除
			console.log(err);
			console.log('down img err==='+_url);

			_isErr = true;

			_self.delRunItem(obj);
			_req.abort();
		})
		.setTimeout(_self.downTimeout,function(){ //设置超时时间
			console.log('down img timeout==='+_url);

			_isErr = true;

			_self.delRunItem(obj);
			_req.abort();
		})
		;
	}
	,addListA:function(_url){//添加有效链接
		var _self = this;
		if(_url){
			_self.aList.push(_url);
		}
	}
	,delListA:function(){ //取出列表第一个并返回
		var _self = this;
		return _self.aList.shift();
	}
	,getImgStep:function(){//递归获取的步骤
		var _self = this,_url,_timer;

		//第一次进来
		if(_self.isInit){
			_url = _self.delListA();
			_self.isInit=false;
		}else{
			//如果播放列表为空
			if(!_self.aList.length){
				if(!_self.nextPage){
					_self.checkRunList();
					return false;
				}else{
					_self.getListPage(_self.nextPage);
					_self.isInit = true;
				}
			}else{
				_url = _self.delListA();
			}
		}

		_self.num++;

		//检查下载列表
		function checkList(){
			var _len = _self._downList.length,_rLen = _self._runList.length;
			if(!_len && !_rLen){ //如果下载列表为0
				if(_self.num<=_self.total && _url){//如果小于最大值并且url不为空
					console.log(_self.num+'===='+_url);
					_self.getImg(_url);
				}
			}else{
				_timer = setTimeout(function(){ //3s 检查一次  根据情况设置
					checkList();
					console.log('wait===_downList='+_len+'|_runList='+_rLen);
				},3000);
			}
		}

		checkList();
	}
	,getListPage:function(_url){
		var _self = this;
		_self.getPage(_url,function(chunk){
			_self.findA(chunk,function(){
				_self.getImgStep();
			});
		});
	}
};

//plugin = require('./plugin/mm8.js');
plugin = require('./plugin/lofter.js');
imgSpider = new spider({
	dir:'download/'
	,mainPage:plugin.url
	,encoding:plugin.encoding
},plugin.rule);
imgSpider.init();
